bitkeeper revision 1.1265 (42435d13hIiIzrasNZHbz13uy4ZTKg)

First attempt at cleanup after merge of shadow code with unstable.

Signed-off-by: michael.fetterman@cl.cam.ac.uk

diff --git a/xen/arch/x86/audit.c b/xen/arch/x86/audit.c
index 97a7baf8bd166d9766c1138f592ccf8f82025371..6d8b2f48a9d89b288fb2efa146d946e7bd44916a 100644 (file)
--- a/xen/arch/x86/audit.c
+++ b/xen/arch/x86/audit.c
@@ -431,8 +431,7 @@ int audit_adjust_pgtables(struct domain *d, int dir, int noisy)
             mfn = page_to_pfn(page);
             page_type = page->u.inuse.type_info & PGT_type_mask;
 
-            if ( page_get_owner(page) != d )
-                BUG();
+            BUG_ON(page_get_owner(page) != d);
 
             page_count++;
 
@@ -563,6 +562,31 @@ int audit_adjust_pgtables(struct domain *d, int dir, int noisy)
 
 #ifndef NDEBUG
 
+void audit_pagelist(struct domain *d)
+{
+    struct list_head *list_ent;
+    int xenpages, totpages;
+
+    list_ent = d->xenpage_list.next;
+    for ( xenpages = 0; (list_ent != &d->xenpage_list); xenpages++ )
+    {
+        list_ent = list_ent->next;
+    }
+    list_ent = d->page_list.next;
+    for ( totpages = 0; (list_ent != &d->page_list); totpages++ )
+    {
+        list_ent = list_ent->next;
+    }
+
+    if ( xenpages != d->xenheap_pages ||
+         totpages != d->tot_pages )
+    {
+        printk("ARGH! dom %d: xen=%d %d, pages=%d %d\n",
+               xenpages, d->xenheap_pages, 
+               totpages, d->tot_pages );
+    }
+}
+
 void _audit_domain(struct domain *d, int flags)
 {
     void scan_for_pfn_in_mfn(struct domain *d, unsigned long xmfn,
@@ -668,6 +692,8 @@ void _audit_domain(struct domain *d, int flags)
 
     spin_lock(&d->page_alloc_lock);
 
+    audit_pagelist(d);
+
     /* PHASE 0 */
 
     list_ent = d->page_list.next;
@@ -679,8 +705,7 @@ void _audit_domain(struct domain *d, int flags)
         mfn = page_to_pfn(page);
         page_type = page->u.inuse.type_info & PGT_type_mask;
 
-        if ( page_get_owner(page) != d )
-            BUG();
+        BUG_ON(page_get_owner(page) != d);
 
         if ( (page->u.inuse.type_info & PGT_count_mask) >
              (page->count_info & PGC_count_mask) )

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 53a3de06b8d82fe7ed7f2ae2268d80a26bbcdd57..d06e9e7f821a4e484239133712c6f2309c1fb7c2 100644 (file)
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -256,6 +256,9 @@ void arch_do_createdomain(struct exec_domain *ed)
         ed->arch.shadow_vtable = __shadow_linear_l2_table;
 
 #ifdef __x86_64__
+        ed->arch.guest_vl3table = __linear_l3_table;
+        ed->arch.guest_vl4table = __linear_l4_table;
+
         d->arch.mm_perdomain_l2 = (l2_pgentry_t *)alloc_xenheap_page();
         memset(d->arch.mm_perdomain_l2, 0, PAGE_SIZE);
         d->arch.mm_perdomain_l2[l2_table_offset(PERDOMAIN_VIRT_START)] = 

diff --git a/xen/arch/x86/mm.c b/xen/arch/x86/mm.c
index 1d79c7bb4b4137c04704a7633cb944962f8c884b..04c3b61e23617e5c74e7334ab8dcb033ff403bb7 100644 (file)
--- a/xen/arch/x86/mm.c
+++ b/xen/arch/x86/mm.c
@@ -1988,48 +1988,58 @@ int do_mmu_update(
     return rc;
 }
 
-void update_shadow_va_mapping(unsigned long va,
-                              unsigned long val,
-                              struct exec_domain *ed,
-                              struct domain *d)
+/* This function assumes the caller is holding the domain's BIGLOCK
+ * and is running in a shadow mode
+ */
+int update_shadow_va_mapping(unsigned long va,
+                             unsigned long val,
+                             struct exec_domain *ed,
+                             struct domain *d)
 {
-    /* This function assumes the caller is holding the domain's BIGLOCK
-     * and is running in a shadow mode
-     */
-
-    unsigned long   sval = 0;
+    unsigned long l1mfn;
+    unsigned long spte;
+    int rc = 0;
 
-    l1pte_propagate_from_guest(d, &val, &sval);
+    check_pagetable(ed, "pre-va"); /* debug */
+    shadow_lock(d);
+        
+    // This is actually overkill - we don't need to sync the L1 itself,
+    // just everything involved in getting to this L1 (i.e. we need
+    // linear_pg_table[l1_linear_offset(va)] to be in sync)...
+    //
+    __shadow_sync_va(ed, va);
 
-    if ( unlikely(__put_user(sval, ((unsigned long *)(
-        &shadow_linear_pg_table[l1_linear_offset(va)])))) )
+#if 1 /* keep check_pagetables() happy */
+    /*
+     * However, the above doesn't guarantee that there's no snapshot of
+     * the L1 table in question; it just says that the relevant L2 and L1
+     * entries for VA are in-sync.  There might still be a snapshot.
+     *
+     * The checking code in _check_pagetables() assumes that no one will
+     * mutate the shadow of a page that has a snapshot.  It's actually
+     * OK to not sync this page, but it seems simpler to:
+     * 1) keep all code paths the same, and
+     * 2) maintain the invariant for _check_pagetables(), rather than try
+     *    to teach it about this boundary case.
+     * So we flush this L1 page, if it's out of sync.
+     */
+    l1mfn = (l2_pgentry_val(linear_l2_table(ed)[l2_table_offset(va)]) >>
+             PAGE_SHIFT);
+    if ( mfn_out_of_sync(l1mfn) )
     {
-        /*
-         * Since L2's are guaranteed RW, failure indicates either that the
-         * page was not shadowed, or that the L2 entry has not yet been
-         * updated to reflect the shadow.
-         */
+        perfc_incrc(extra_va_update_sync);
+        __shadow_sync_mfn(d, l1mfn);
+    }
+#endif /* keep check_pagetables() happy */
 
-        /* Can't use linear_l2_table with external tables. */
-        BUG_ON(shadow_mode_external(current->domain));
+    if ( unlikely(__put_user(val, &l1_pgentry_val(
+                                 linear_pg_table[l1_linear_offset(va)]))) )
+        return -EINVAL;
 
-        l2_pgentry_t gpde = linear_l2_table[l2_table_offset(va)];
-        unsigned long gpfn = l2_pgentry_val(gpde) >> PAGE_SHIFT;
+    // also need to update the shadow
 
-        if (get_shadow_status(d, gpfn))
-        {
-            unsigned long gmfn = __gpfn_to_mfn(d, gpfn);
-            unsigned long *gl1e = map_domain_mem(gmfn << PAGE_SHIFT);
-            unsigned l1_idx = l1_table_offset(va);
-            gl1e[l1_idx] = sval;
-            unmap_domain_mem(gl1e);
-            put_shadow_status(d);
-
-            perfc_incrc(shadow_update_va_fail1);
-        }
-        else
-            perfc_incrc(shadow_update_va_fail2);
-    }
+    l1pte_propagate_from_guest(d, val, &spte);
+    shadow_set_l1e(va, spte, 0);
 
     /*
      * If we're in log-dirty mode then we need to note that we've updated
@@ -2037,9 +2047,12 @@ void update_shadow_va_mapping(unsigned long va,
      * for this.
      */
     if ( shadow_mode_log_dirty(d) )
-        mark_dirty(d, va_to_l1mfn(va));
+        mark_dirty(d, va_to_l1mfn(ed, va));
+
+    shadow_unlock(d);
+    check_pagetable(ed, "post-va"); /* debug */
 
-    check_pagetable(d, ed->arch.guest_table, "va"); /* debug */
+    return rc;
 }
 
 int update_grant_va_mapping(unsigned long va,
@@ -2104,8 +2117,21 @@ int do_update_va_mapping(unsigned long va,
      * XXX When we make this support 4MB superpages we should also deal with 
      * the case of updating L2 entries.
      */
-    if ( unlikely(!shadow_mode_enabled(d)) )
+    if ( unlikely(shadow_mode_enabled(d)) )
+    {
+        if ( unlikely(percpu_info[cpu].foreign &&
+                      (shadow_mode_translate(d) ||
+                       shadow_mode_translate(percpu_info[cpu].foreign))) )
+        {
+            // The foreign domain's pfn's are in a different namespace.
+            // There's not enough information in just a gpte to figure out
+            // how to (re-)shadow this entry.
+            //
+            domain_crash();
+        }
+    
         rc = update_shadow_va_mapping(va, val, ed, d);
+    }
     else if ( unlikely(!mod_l1_entry(&linear_pg_table[l1_linear_offset(va)],
                                      mk_l1_pgentry(val))) )
         rc = -EINVAL;
@@ -2484,7 +2510,7 @@ void ptwr_flush(const int which)
 
     if ( which == PTWR_PT_ACTIVE )
     {
-        pl2e = &linear_l2_table(ed)[ptwr_info[cpu].ptinfo[which].l2_idx];
+        pl2e = &__linear_l2_table[ptwr_info[cpu].ptinfo[which].l2_idx];
         *pl2e = mk_l2_pgentry(l2_pgentry_val(*pl2e) | _PAGE_PRESENT); 
     }
 
@@ -2502,9 +2528,9 @@ static int ptwr_emulated_update(
     unsigned int bytes,
     unsigned int do_cmpxchg)
 {
-    unsigned long sstat, pte, pfn;
+    unsigned long pte, pfn;
     struct pfn_info *page;
-    l1_pgentry_t ol1e, nl1e, *pl1e, *sl1e;
+    l1_pgentry_t ol1e, nl1e, *pl1e;
     struct domain *d = current->domain;
 
     /* Aligned access only, thank you. */
@@ -2581,6 +2607,8 @@ static int ptwr_emulated_update(
     /* Propagate update to shadow cache. */
     if ( unlikely(shadow_mode_enabled(d)) )
     {
+        BUG(); // XXX fix me...
+#if 0
         sstat = get_shadow_status(d, page_to_pfn(page));
         if ( sstat & PSH_shadowed )
         {
@@ -2590,6 +2618,7 @@ static int ptwr_emulated_update(
                 d, &l1_pgentry_val(nl1e), &l1_pgentry_val(*sl1e));
             unmap_domain_mem(sl1e);
         }
+#endif
     }
 
     /* Finally, drop the old PTE. */
@@ -2636,14 +2665,11 @@ int ptwr_do_page_fault(unsigned long addr)
     // not supported in combination with various shadow modes!
     ASSERT( !shadow_mode_enabled(ed->domain) );
 
-    /* Can't use linear_l2_table with external tables. */
-    BUG_ON(shadow_mode_external(ed->domain));
-
     /*
      * Attempt to read the PTE that maps the VA being accessed. By checking for
      * PDE validity in the L2 we avoid many expensive fixups in __get_user().
      */
-    if ( !(l2_pgentry_val(linear_l2_table(ed)[addr>>L2_PAGETABLE_SHIFT]) &
+    if ( !(l2_pgentry_val(__linear_l2_table[addr>>L2_PAGETABLE_SHIFT]) &
            _PAGE_PRESENT) ||
          __get_user(pte, (unsigned long *)
                     &linear_pg_table[l1_linear_offset(addr)]) )
@@ -2684,7 +2710,7 @@ int ptwr_do_page_fault(unsigned long addr)
      * Is the L1 p.t. mapped into the current address space? If so we call it
      * an ACTIVE p.t., otherwise it is INACTIVE.
      */
-    pl2e = &linear_l2_table(ed)[l2_idx];
+    pl2e = &__linear_l2_table[l2_idx];
     l2e  = l2_pgentry_val(*pl2e);
     which = PTWR_PT_INACTIVE;
     if ( (l2e >> PAGE_SHIFT) == pfn )
@@ -2824,31 +2850,6 @@ void ptwr_status(void)
     page = &frame_table[pfn];
 }
 
-void audit_pagelist(struct domain *d)
-{
-    struct list_head *list_ent;
-    int xenpages, totpages;
-
-    list_ent = d->xenpage_list.next;
-    for ( xenpages = 0; (list_ent != &d->xenpage_list); xenpages++ )
-    {
-        list_ent = list_ent->next;
-    }
-    list_ent = d->page_list.next;
-    for ( totpages = 0; (list_ent != &d->page_list); totpages++ )
-    {
-        list_ent = list_ent->next;
-    }
-
-    if ( xenpages != d->xenheap_pages ||
-         totpages != d->tot_pages )
-    {
-        printk("ARGH! dom %d: xen=%d %d, pages=%d %d\n",
-               xenpages, d->xenheap_pages, 
-               totpages, d->tot_pages );
-    }
-}
-
 #endif /* NDEBUG */
 
 /*

diff --git a/xen/arch/x86/shadow.c b/xen/arch/x86/shadow.c
index 171b10ec61eb829f87bdda0af636a56e70b56160..fe9a009e17baa125f0324d71174f16ba86f311c1 100644 (file)
--- a/xen/arch/x86/shadow.c
+++ b/xen/arch/x86/shadow.c
@@ -1245,11 +1245,11 @@ void vmx_shadow_clear_state(struct domain *d)
 }
 
 unsigned long
-gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn)
+translate_gpfn_to_mfn(struct domain *d, unsigned long gpfn)
 {
     ASSERT( shadow_mode_translate(d) );
 
-    perfc_incrc(gpfn_to_mfn_safe);
+    perfc_incrc(translate_gpfn_to_mfn);
 
     unsigned long va = gpfn << PAGE_SHIFT;
     unsigned long phystab = pagetable_val(d->arch.phys_table);
@@ -1258,7 +1258,7 @@ gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn)
     unmap_domain_mem(l2);
     if ( !(l2_pgentry_val(l2e) & _PAGE_PRESENT) )
     {
-        printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => 0 l2e=%p\n",
+        printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => 0 l2e=%p\n",
                d->id, gpfn, l2_pgentry_val(l2e));
         return INVALID_MFN;
     }
@@ -1267,12 +1267,14 @@ gpfn_to_mfn_safe(struct domain *d, unsigned long gpfn)
     l1_pgentry_t l1e = l1[l1_table_offset(va)];
     unmap_domain_mem(l1);
 
-    printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => %p phystab=%p l2e=%p l1tab=%p, l1e=%p\n",
+#if 0
+    printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => %p phystab=%p l2e=%p l1tab=%p, l1e=%p\n",
            d->id, gpfn, l1_pgentry_val(l1e) >> PAGE_SHIFT, phystab, l2e, l1tab, l1e);
+#endif
 
     if ( !(l1_pgentry_val(l1e) & _PAGE_PRESENT) )
     {
-        printk("gpfn_to_mfn_safe(d->id=%d, gpfn=%p) => 0 l1e=%p\n",
+        printk("translate_gpfn_to_mfn(d->id=%d, gpfn=%p) => 0 l1e=%p\n",
                d->id, gpfn, l1_pgentry_val(l1e));
         return INVALID_MFN;
     }

diff --git a/xen/include/asm-x86/domain.h b/xen/include/asm-x86/domain.h
index 8e21db99163c29fd15197c4f9907bc36ce1cc5a6..3c37d78d6a33eb3b67b96fb044d01571d67b9768 100644 (file)
--- a/xen/include/asm-x86/domain.h
+++ b/xen/include/asm-x86/domain.h
@@ -127,6 +127,11 @@ struct arch_exec_domain
     l2_pgentry_t *monitor_vtable;              /* virtual address of monitor_table */
     l1_pgentry_t *hl2_vtable;                  /* virtual address of hl2_table */
 
+#ifdef __x86_64__
+    l3_pgentry_t *guest_vl3table;
+    l4_pgentry_t *guest_vl4table;
+#endif
+
     unsigned long monitor_shadow_ref;
 
     /* Virtual CR2 value. Can be read/written by guest. */

diff --git a/xen/include/asm-x86/mm.h b/xen/include/asm-x86/mm.h
index 65d9426c26a18a53a191f96203c92d7cc83dfd53..4c03e8189e72aaf1a890af75c80fff060bf0d8b7 100644 (file)
--- a/xen/include/asm-x86/mm.h
+++ b/xen/include/asm-x86/mm.h
@@ -4,6 +4,7 @@
 
 #include <xen/config.h>
 #include <xen/list.h>
+#include <xen/sched.h>
 #include <asm/io.h>
 
 /*
@@ -180,12 +181,13 @@ static inline int get_page(struct pfn_info *page,
              unlikely((nx & PGC_count_mask) == 0) || /* Count overflow? */
              unlikely(d != _domain) )                /* Wrong owner? */
         {
-          if ( !domain->arch.shadow_mode )
-            DPRINTK("Error pfn %p: rd=%p(%d), od=%p(%d), caf=%08x, taf=%08x\n",
-                    page_to_pfn(page), domain, (domain ? domain->id : -1),
-                    page_get_owner(page),
-                    (page_get_owner(page) ? page_get_owner(page)->id : -1),
-                    x, page->u.inuse.type_info);
+            if ( !domain->arch.shadow_mode )
+                DPRINTK("Error pfn %p: rd=%p(%d), od=%p(%d), caf=%08x, "
+                        "taf=%08x\n",
+                        page_to_pfn(page), domain, (domain ? domain->id : -1),
+                        page_get_owner(page),
+                        (page_get_owner(page) ? page_get_owner(page)->id : -1),
+                        x, page->u.inuse.type_info);
             return 0;
         }
         __asm__ __volatile__(

diff --git a/xen/include/asm-x86/shadow.h b/xen/include/asm-x86/shadow.h
index 625542b26bcf17ee99213ebaa02d9710403b7bff..739f65e48f7974b6c6ed49d3ee07f1f58bf93ad5 100644 (file)
--- a/xen/include/asm-x86/shadow.h
+++ b/xen/include/asm-x86/shadow.h
@@ -195,9 +195,7 @@ static inline void shadow_mode_disable(struct domain *d)
       ? translate_gpfn_to_mfn(_d, gpfn)                \
       : (gpfn) )
 
-#define translate_gpfn_to_mfn gpfn_to_mfn_safe
-
-extern unsigned long gpfn_to_mfn_safe(
+extern unsigned long translate_gpfn_to_mfn(
     struct domain *d, unsigned long gpfn);
 
 /************************************************************************/
@@ -659,12 +657,11 @@ static inline void hl2e_propagate_from_guest(
         if ( unlikely((current->domain != d) && !shadow_mode_external(d)) )
         {
             // Can't use __gpfn_to_mfn() if we don't have one of this domain's
-            // page tables currently installed.  What a pain in the neck!
-            //
+            // page tables currently installed.
             // This isn't common -- it only happens during shadow mode setup
             // and mode changes.
             //
-            mfn = gpfn_to_mfn_safe(d, pfn);
+            mfn = translate_gpfn_to_mfn(d, pfn);
         }
         else
             mfn = __gpfn_to_mfn(d, pfn);

diff --git a/xen/include/xen/perfc_defn.h b/xen/include/xen/perfc_defn.h
index 2e776841e307a957db43722f788f48f449ad18d9..318882a670bac94db8bc06eb4ec6deadf6198541 100644 (file)
--- a/xen/include/xen/perfc_defn.h
+++ b/xen/include/xen/perfc_defn.h
@@ -98,7 +98,7 @@ PERFCOUNTER_CPU(shadow_get_page_fail,   "shadow_get_page_from_l1e fails" )
 PERFCOUNTER_CPU(validate_hl2e_calls,               "calls to validate_hl2e_change")
 PERFCOUNTER_CPU(validate_hl2e_changes,             "validate_hl2e makes changes")
 PERFCOUNTER_CPU(exception_fixed,                   "pre-exception fixed")
-PERFCOUNTER_CPU(gpfn_to_mfn_safe,                  "calls to gpfn_to_mfn_safe")
+PERFCOUNTER_CPU(translate_gpfn_to_mfn,             "calls to translate_gpfn_to_mfn")
 PERFCOUNTER_CPU(remove_write_access,               "calls to remove_write_access")
 PERFCOUNTER_CPU(remove_write_access_easy,          "easy outs of remove_write_access")
 PERFCOUNTER_CPU(remove_write_no_work,              "no work in remove_write_access")